package com.fll.main;

import com.alibaba.fastjson.JSONObject;
import com.fll.config.FinalFieldConfig;
import com.fll.config.UrlConfig;
import com.fll.entity.CourseCatalogModel;
import com.fll.entity.CourseModel;
import com.fll.entity.result.ResulEntity;
import com.fll.util.CrawUtil;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

/**
 * @Description
 * @Author fanlei
 * @Version 1.0
 * @Date 2018年09月12日
 */
public class Test {

    private static Logger logger = LoggerFactory.getLogger(Test.class);

    public static void main(String[] s){
        List<String> ll = new ArrayList<String>();

        /**
         * 注意每个url
         *
         * 自动爬取分页，
         *
         */
//ll.add("https://www.jikexueyuan.com/course/hadoop/");
//ll.add("https://www.jikexueyuan.com/course/zookeeper/");
//ll.add("https://www.jikexueyuan.com/course/docker/");
//ll.add("https://www.jikexueyuan.com/course/kafka/");
//ll.add("https://www.jikexueyuan.com/course/storm/");
//ll.add("https://www.jikexueyuan.com/course/hive/");
//ll.add("https://www.jikexueyuan.com/course/flume/");
//ll.add("https://www.jikexueyuan.com/course/hbase/");
//ll.add("https://www.jikexueyuan.com/course/spark/");
//ll.add("https://www.jikexueyuan.com/course/mongoDB/");
//ll.add("https://www.jikexueyuan.com/course/mysql/");
//ll.add("https://www.jikexueyuan.com/course/nginx/");
//ll.add("https://www.jikexueyuan.com/course/jsp/");
ll.add("https://www.jikexueyuan.com/course/maven/");
//ll.add("https://www.jikexueyuan.com/course/spring/");
//ll.add("https://www.jikexueyuan.com/course/springmvc/");
//ll.add("https://www.jikexueyuan.com/course/mybatis/");
//ll.add("https://www.jikexueyuan.com/course/hibernate/");
//ll.add("https://www.jikexueyuan.com/course/lucene/");
//ll.add("https://www.jikexueyuan.com/course/j2eeframe/");
//ll.add("https://www.jikexueyuan.com/course/j2eepractice/");
//ll.add("https://www.jikexueyuan.com/course/j2eepractice/");


        Test test = new Test();
        //获取下载信息
        List<CourseModel> courseModels = test.getCourseModelByUrl(ll.get(0), null);
        for (CourseModel model :
                courseModels) {
            model.setCourseDetailModels(test.crawCourseCatalogModel(model.getCourseId()));
        }

        CrawUtil.serializer(FinalFieldConfig.infoPath,courseModels);

        //开始下载
        test.downloadMp4(courseModels);


    }


    public List<CourseModel> getCourseModelByUrl(String url,Integer page){
        if (page != null){
            url = String.format(UrlConfig.pagingUrl, url, page);
        }
        List<CourseModel> courseModels = crawlCourseModel(url);
        if (courseModels.size() != 0){
            Integer currPage = courseModels.get(courseModels.size() - 1).getCurrPage();
            Integer total = courseModels.get(courseModels.size() - 1).getTotal();
            if (currPage != total){
                courseModels.addAll(getCourseModelByUrl(url,currPage));
            }
        }
        return courseModels;
    }


    /**
     * 获取课程模块
     * @param url
     * @return
     */
    public List<CourseModel> crawlCourseModel(String url) {
        logger.info("开始打开网页并解析JavaScript-->{}",url);
        Document document = createDocumentByWebClient(url);
        //选择每一个课程大模块
        Elements select = document.select("div.lesson-list ul li");
        Iterator<Element> iterator = select.iterator();
        List<CourseModel> list = new ArrayList<CourseModel>();
        CourseModel course = null;
        List<String> parentNames = null;
        Integer currPage = null;
        Integer total = null;
        while (iterator.hasNext()){
            Element next = iterator.next();

            String type = next.select("div.lessonimg-box i").get(0).attr("class");
            if(FinalFieldConfig.downType.contains(type)){//满足下载的条件
                course = new CourseModel();
                //获取每个大模块的类型
                course.setType(type);
            }else {
                continue;
            }

            //爬取父课程库的名字
            if(parentNames == null){
                parentNames = new ArrayList<String>();
                Elements select1 = document.select("div.crumbs div a");
                Iterator<Element> iterator1 = select1.iterator();
                while (iterator1.hasNext()){
                    Element next1 = iterator1.next();
                    parentNames.add(CrawUtil.StrDispose(next1.html()));
                }
            }
            course.setParentName(parentNames);

            //获取每个大模块的名字
            course.setName(CrawUtil.StrDispose(next.select("div.lesson-infor h2.lesson-info-h2 a").get(0).html()));


//            System.out.print(next.getElementById("2436"));
            //获取每个大模块的url
            course.setCourseUrl("https:" + next.select("div.lessonimg-box a").get(0).attr("href"));

            //课程id
            course.setCourseId(course.getCourseUrl().substring(course.getCourseUrl().indexOf("course/")+7, course.getCourseUrl().lastIndexOf(".html")));//www.jikexueyuan.com/course/2332.html

            if(currPage == null || total == null){
                Elements select1 = document.select("div#page-nav ul li");
                if(select1.size() == 0){
                    currPage = 1;
                    total = 1;
                }else {
                    currPage = Integer.valueOf(select1.select(".pgCurrent").html());
                    total = CrawUtil.IntByStr(select1.select(".pagetotal").html());
                }
            }

            //模块所在的页数
            course.setCurrPage(currPage);

            //总页数
            course.setTotal(total);

            logger.info("此模块在第{}页，共{}页",currPage,total);
            logger.info("获取《{}》信息成功",course.getName());

            list.add(course);
        }
        return list;
    }


    //获取视频模块
    public List<CourseCatalogModel> crawCourseCatalogModel(String courseId){
        String url = String.format(UrlConfig.courseUrl,courseId,1);
        logger.info("开始打开课程（ID:{}）网页并解析JavaScript-->{}",courseId,url);
        Document document = createDocument(url);
        //选择大模块
        Elements select = document.select("div.video-list div.lesson-box ul li");
        //课程章节总数
        Integer sum = select.size();
        logger.info("课程（ID:{}）总的章节数{}",courseId,sum);

        Iterator<Element> iterator = select.iterator();
        List<CourseCatalogModel> list = new ArrayList<CourseCatalogModel>();
        CourseCatalogModel courseCatalog = null;
        while (iterator.hasNext()){

            Element next = iterator.next();
            courseCatalog = new CourseCatalogModel();

            //此课程章节的序号
            courseCatalog.setSort(Integer.valueOf(next.select("i.lessonmbers em").get(0).html()));
            logger.info("获取课程ID:{}第{}条课程列表",courseId,courseCatalog.getSort());

            //课程章节总数
            courseCatalog.setSum(sum);

            //课程目录url
            courseCatalog.setCourseCatalogUrl(String.format(UrlConfig.courseUrl, courseId, courseCatalog.getSort()));

            //获取名字
            Elements nameSelect = next.select("div.text-box h2 a");
            if(nameSelect == null || nameSelect.size() == 0){
                courseCatalog.setName(document.select("span.tit").get(0).html());
            }else {
                courseCatalog.setName(CrawUtil.StrDispose(nameSelect.get(0).html()));
            }

            //课程时间
            courseCatalog.setTime(next.select("div.text-box h2 p").get(0).html());

            //获取描述
            courseCatalog.setDescription(next.select("div.text-box p").get(1).html());

            //获取查询视屏下载的url
            courseCatalog.setSelectVideoDowbloadUrl(String.format(UrlConfig.courseDownload,courseCatalog.getSort(),courseId));

            //查询视频下载链接
            ResulEntity resultVo = queryVideoDownloadUrl(courseCatalog.getSelectVideoDowbloadUrl());

            //视频下载url
            courseCatalog.setVideoDownloadUrl(resultVo.getData().getUrls());

            //资源下载地址
            courseCatalog.setResourceDownloadUrl(String.format(UrlConfig.courseResourceDownload,courseId));

            logger.info("获取课程ID{}第{}条信息成功",courseId,courseCatalog.getSort());

            list.add(courseCatalog);
        }

        return list;
    }

    /**
     * 获取真的下载视频链接
     * @param url
     * @return
     */
    public ResulEntity queryVideoDownloadUrl(String url){
        Connection connection = createAuthConnection(url,true).validateTLSCertificates(false).ignoreContentType(true);
        Connection.Response execute = null;
        try {
            execute = connection.execute();
        } catch (IOException e) {
            e.printStackTrace();
        }
        String s = execute.body();

        return JSONObject.parseObject(s, ResulEntity.class);
    }


    //下载视频
    public void downloadMp4(List<CourseModel> modelBeans){

        //重新执行时删除之前成功的大模块
        CrawUtil.deleteFile(FinalFieldConfig.outFilePath + FinalFieldConfig.succePath);

        for (CourseModel modelBean:
                modelBeans) {
            String cellPath = CrawUtil.joinStr(modelBean.getParentName(),modelBean.getType(),modelBean.getName());
            CrawUtil.mkdirsFile(cellPath);
            for (CourseCatalogModel courseDetail :
                    modelBean.getCourseDetailModels()) {
                exec(courseDetail,cellPath,modelBean.getType());
            }

            //反序列化
            List<CourseModel> o = (List<CourseModel>)CrawUtil.reSerializer(FinalFieldConfig.succePath);
            if (o == null || o.size() == 0){
                o = new ArrayList<CourseModel>();
                o.add(modelBean);
            }else {
                o.add(modelBean);
            }
            //下载成功的大模块序列化记录
            CrawUtil.serializer(FinalFieldConfig.succePath,o);
        }

    }

    public void exec(CourseCatalogModel courseDetail,String cellPath,String type){
        FileOutputStream outMp4 = null;
        FileOutputStream outRes = null;
        FileOutputStream outDesc = null;
        try {
            if(courseDetail.getVideoDownloadUrl() == null){
                return;
            }

            logger.info("开始下载---->{}.mp4",courseDetail.getName());

//            Connection.Response execute = Jsoup.connect(courseDetail.getVideoDownloadUrl())
//                    .ignoreContentType(true)
//                    .maxBodySize(1024*1024*1024)
//                    .execute();//最大下载1G
//
//            Long size = Long.valueOf(execute.headers().get("Content-Length"));
//
//            outMp4 = new FileOutputStream(new File(cellPath + "/" + courseDetail.getSort() + "-"  + courseDetail.getName() + ".mp4"));
//            outMp4.write(execute.bodyAsBytes());

            down(courseDetail.getName(),courseDetail.getVideoDownloadUrl(),cellPath + "/" + courseDetail.getSort() + "-"  + courseDetail.getName() + ".mp4");

            File file = new File(cellPath + "/资料.zip");
            if(courseDetail.getResourceDownloadUrl() != null && !file.exists()) {//不存在
                Connection.Response execute1 = Jsoup.connect(courseDetail.getResourceDownloadUrl())
                        .ignoreContentType(true)
                        .maxBodySize(1024*1024*1024)
                        .execute();//最大下载1G
                outRes = new FileOutputStream(new File(cellPath + "/资料.zip"));
                outRes.write(execute1.bodyAsBytes());
            }

            outDesc = new FileOutputStream(new File(cellPath + "/" + courseDetail.getSort() + "-"  + courseDetail.getName() + ".txt"));
            outDesc.write(("时长：" + courseDetail.getTime() + "\\r\\n" + courseDetail.getDescription()).getBytes());
        }catch (Exception e){
            e.printStackTrace();
            throw new RuntimeException(e.getMessage());
        }finally {
            try {
                if (outDesc != null){
                    outDesc.flush();
                    outDesc.close();
                }
                if (outRes != null){
                    outRes.flush();
                    outRes.close();
                }
                if (outMp4 != null){
                    outMp4.flush();
                    outMp4.close();
                }
            }catch (Exception e){
                e.printStackTrace();
            }
            logger.info("下载完毕---->{}.mp4",courseDetail.getName());
        }
    }


    public void down(String name,String downloadUrl,String path){
        URL url = null;
        try {
            url = new URL(downloadUrl);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();

            if (conn.getResponseCode() == 200){
                InputStream is = conn.getInputStream();
                OutputStream out = new FileOutputStream(path);
                byte[] buffer = new byte[1024*1024];

                Double size = Double.valueOf(conn.getHeaderField("Content-Length"));

                int len = -1;
                long startTime = System.currentTimeMillis();
                long endTime = System.currentTimeMillis();
                long dataLen = 0;
                while((len = is.read(buffer)) != -1){
                    dataLen += len;
                    out.write(buffer, 0, len);
                    if ((endTime = System.currentTimeMillis()) - startTime > 1000) {
                        logger.info("{} 下载完成---->{}%",name,String.format("%.2f", (dataLen/size)*100.0));
                        startTime = endTime;
                    }
                }
                logger.info("{} 下载完成---->{}",name,"100%");
                is.close();
                out.flush();
                out.close();
            }

        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }finally {

        }
    }



    public Document webClient(String url) throws IOException {
        WebClient wc = new WebClient(BrowserVersion.CHROME);
        wc.getOptions().setUseInsecureSSL(true);
        wc.getOptions().setJavaScriptEnabled(true); // 启用JS解释器，默认为true
        wc.getOptions().setCssEnabled(false); // 禁用css支持
        wc.getOptions().setThrowExceptionOnScriptError(false); // js运行错误时，是否抛出异常
        wc.getOptions().setTimeout(300000); // 设置连接超时时间 ，这里是30S。如果为0，则无限期等待
        wc.getOptions().setDoNotTrackEnabled(false);
        HtmlPage page = wc.getPage(url);
        return Jsoup.parse(page.asXml());
    }

    /**
     * 创建Document执行javascript
     * @param url
     * @return
     */
    public Document createDocumentByWebClient(String url){
//        Connection connection = createConnection(url,false).validateTLSCertificates(false);
        //解析html
        try {
            return Jsoup.parse(webClient(url).outerHtml());
        } catch (IOException e) {
            e.printStackTrace();
            throw new RuntimeException(e.getMessage());
        }
    }

    public Document createDocument(String url){
        Connection connection = createAuthConnection(url,false).validateTLSCertificates(false);
        //解析html
        try {
            return Jsoup.parse(connection.get().outerHtml());
        } catch (IOException e) {
            e.printStackTrace();
            throw new RuntimeException(e.getMessage());
        }
    }

    public Connection createAuthConnection(String url,boolean isHeaders){
        Connection connect = Jsoup.connect(url).ignoreContentType(false);
        if(isHeaders){
            connect.headers(FinalFieldConfig.map);
        }
        return connect;
    }

}
